# -*- coding: UTF-8 -*-
import os
import socket
import threading
import urllib2, urllib, re
import threadreturnhelp

__author__ = 'beibei'


def scranproxy(url, haschile=True):
    user_agent = ''
    # url = "http://www.youdaili.net/Daili/http/35287.html";
    values = {'name': 'Michael Foord',
              'location': 'pythontab',
              'language': 'Python'}
    # 将正则表达式编译成Pattern对象
    headers = {'User-Agent': user_agent}
    data = urllib.urlencode(values)
    req = urllib2.Request(url, data=None, headers=headers)
    response = urllib2.urlopen(req)
    the_page = response.read()

    pattern = re.compile(
        r"((?:(?:25[0-5]|2[0-4]\d|(?:(?:1\d{2})|(?:[1-9]?\d)))\.){3}(?:25[0-5]|2[0-4]\d|(?:(?:1\d{2})|(?:[1-9]?\d))))\:([\d]{0,4})",
        re.X)
    # 使用Pattern匹配文本，获得匹配结果，无法匹配时将返回None
    ipports = pattern.findall(the_page)
    if haschile:  # 分页信息
        math = re.search(r'<div class="pagebreak">(.*?)</div>', the_page)
        if (math):
            str = math.group()
            allpages = re.findall(r'[\d]+_[\d]{0,2}\.html', str)
            allpages = list(set(allpages))
            print allpages
            for page in allpages:
                print page
                basepath = url[0:url.rindex('/') + 1]
                childipports = scranproxy(basepath + page, False)
                ipports.extend(childipports)
    # ipports = list(set(ipports))
    # print 'sorting'
    ipports = {}.fromkeys(ipports).keys()
    return ipports



    # for ipprot in ipports:
    # 使用Match获得分组信息
    #    print ipprot
    # response=urllib2.urlopen(url)
    # txt=response.read()
    # print(the_page)


def saveproxy(proxys=[]):
    '''
    保存 代理列表
    :param proxys: 代理列表
    :return:
    '''

    realpath = os.path.dirname(os.path.realpath(__file__))
    filename = "ips.txt"
    savefile = realpath + "/" + filename
    txt = "";
    for p in proxys:
        txt = txt + p[0] + ":" + str(p[1]) + "\r\n"

    print 'startSavefile Path:' + savefile

    f = open(savefile, 'w')
    f.write(txt)
    f.close()
    pass


def initproxy():
    '''
    初始化 代理
    :return:
    '''
    realpath = os.path.dirname(os.path.realpath(__file__))
    filename = "ips.txt"
    savefile = realpath + "/" + filename
    txt = "";
    f = open(savefile, 'r')
    lns = f.readlines()
    f.close()
    proxys = []
    #lns = txt.split('\r\n')

    for ln in lns:
        if len(ln) > 0:
            arr = ln.split(':')
            #print(len(proxys))
            arr[1] = int(arr[1])
            proxys.append(tuple(arr))
    return proxys
    pass

def replacerealproxy(proxys):
    socket.setdefaulttimeout(1)
    realproxys=[]
    co=CheckOpen(proxys,5)
    #print co._num,len(proxys)
    threads=[]

    for i in range(1,co._num+1):
        t=threading.Thread(target=co.run,args=(i,))
        threads.append(t)
        t.start()
        #co.run(i)
    for thread in threads:
        thread.join()

    realproxys=co._realproxys

    #for proxy in proxys:
        #if(portisopen(proxy[0],proxy[1])):
        #   realproxys.append(proxy)
    return realproxys


class CheckOpen:

    def __init__(self,proxys=[],size=5):
        self._proxys=proxys
        self._size=size
        self._num=0
        self._realproxys=[]
        if(len(self._proxys)%size==0):
            self._num=len(self._proxys)/size
        else:
            self._num=len(self._proxys)/size+1

    def getnum(self):
        return self._num

    def run(self,curnum):
        if(curnum>self._num): return
        startindex=(curnum-1)*self._size
        endindex=startindex+self._size
        if(endindex>=len(self._proxys)):endindex=len(self._proxys)
        list=self._proxys[startindex: endindex]
        #print startindex,endindex
        for proxy in list:
            if self.portisopen(proxy[0],proxy[1]):
                self._realproxys.append(proxy)
        #print curnum
        #print list
        pass

    def portisopen(self, tgtHost, tgtPort):
        try:
            connSkt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            connSkt.connect((tgtHost, tgtPort))
            return True;
        except:
            return False;
        finally:
            connSkt.close()

if __name__ == '__main__':
    #ipports=scranproxy("http://www.youdaili.net/Daili/QQ/36712.html",True)
    #print ipports
    #saveproxy(ipports)

    ipports = initproxy()
    print ipports
    realproxy=replacerealproxy(ipports)
    print realproxy
    print len(realproxy)
    saveproxy(realproxy)

